# Scrapy settings for xiaodian project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'xiaodian'

SPIDER_MODULES = ['xiaodian.spiders']
NEWSPIDER_MODULE = 'xiaodian.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'xiaodian (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 0.2
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'xiaodian.middlewares.XiaodianSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
  #  'xiaodian.middlewares.XiaodianDownloaderMiddleware': 543,
  'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550
}

RETRY_ENABLE=True
RETRY_TIMES=10
RETRY_HTTP_CODECS = [403]
HTTPERROR_ALLOWED_CODES = [403] #当遇到403的时候爬虫脚本不退出

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'xiaodian.pipelines.MysqlDpInfoPipeline': 300
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

#设置日志级别： DEBUG < INFO < WARNING < ERROR < CRITICAL
#日志需要自己添加，配置文件中没有，在空白处添加即可
LOGGER_OPTION = {'level': 'DEBUG', 'format': '{time:YYYY-MM-DD HH:mm:ss} - {level} - {file} - {line} - {message}',
                 'rotation': "10 MB", 'encoding': 'utf-8', 'enqueue': True}
import os
BASE_PATH = os.path.abspath(os.path.dirname(__file__))
WRITE_LOG = False
LOG_LEVEL='DEBUG'
#定义日志输出文件
LOG_FILE='spider.log'
#设置导出数据的编码格式
FEED_EXPORT_ENCODING='utf-8'
#在配置文件末尾添加mysql常用变量
MYSQL_HOST='db-mysql.dian-stable.com'
MYSQL_USER='xd_spider'
MYSQL_PASSWORD='pWc9kdZ2n8afUlV0'
MYSQL_PORT = 3306
MYSQL_DB_NAME='xd_spider'
MYSQL_CHARSET='utf8'

DB_OPTION = {'host': 'db-mysql.dian-stable.com', 'user': 'xd_spider', 'password': 'pWc9kdZ2n8afUlV0',
                 'db': 'xd_spider', 'port': 3306}

SPIDERS_DIR = 'spiders'
PROXY_API = 'http://pool.apps.dian-int.com/api/get_ip'

COOKIES = {
  'dianping': "_lxsdk_cuid=17e2f12c859c8-05ad5c61137d07-4c607a68-1fa400-17e2f12c859c8; _hc.v=2bab1e8c-f819-18b7-48c3-054b005a5c94.1641467267; s_ViewType=10; ua=dpuser_1588870495; ctu=986146f4cd7c33a9bd3dae5f9217b886066d9610e500b8a95eb70158403ae7f0; aburl=1; switchcityflashtoast=1; _ga=GA1.2.2106161433.1646637106; cityid=3; uuid=CA10578CAEC9A351320A3E3D6B22F641977D843EB15CBFF268F8DC6955EF4E45; iuuid=CA10578CAEC9A351320A3E3D6B22F641977D843EB15CBFF268F8DC6955EF4E45; _lxsdk=CA10578CAEC9A351320A3E3D6B22F641977D843EB15CBFF268F8DC6955EF4E45; Hm_lvt_dbeeb675516927da776beeb1d9802bd4=1654849711; cy=1; cye=shanghai; citypinyin=hangzhou; cityname=5p2t5bee; m_flash2=1; Hm_lvt_602b80cf8079ae6591966cc70a3940e7=1655951640,1656473312,1656554190,1657072395; Hm_lpvt_602b80cf8079ae6591966cc70a3940e7=1657072395; _lxsdk_s=181d136244d-52c-f3d-175%7C%7C2"
}
